# -*- coding: utf-8 -*-
import requests
from lxml import etree
from bs4 import BeautifulSoup
import re
import time
import traceback
from loguru import logger
import random
import datetime
import execjs
import threading
import sys
import os
import json
import urllib.parse
from fake_useragent import UserAgent
import concurrent.futures
from queue import Queue
from src.service.requests_middler import requests_sys
from src.service.requests_middler import proxy_requests_sys
from src.service.requests_middler import insert_shipping_info
from src.service.requests_middler import read_excel
from src.service.requests_middler import read_transport_json
from src.service.requests_middler import get_proxies

current_working_directory = os.getcwd()
ua = UserAgent()
random_user_agent = ua.random
stop_threading_event = threading.Event()

total_requests = 0
loss_requests = 0

ip_queue = Queue()


def proxies_queue():
    # 获取代理并放入队列
    if ip_queue.empty():
        proxy = get_proxies()
        logger.info(f"获取ip代理成功: {proxy}")
        if proxy:
            ip_queue.put(proxy)
        else:
            logger.error(f"获取ip代理失败")
    else:
        proxy = ip_queue.get()
        return proxy


# 解码函数
def decode(data):
    # 解码
    decoded_data = urllib.parse.unquote(data.encode('utf-8').decode('unicode_escape'))
    parsed_data = json.loads(decoded_data)
    return parsed_data


def get_point(query):
    global total_requests
    global loss_requests
    headers = {
        "Accept": "text/javascript,text/html,application/xml,text/xml,*/*",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Content-Type": "application/x-www-form-urlencoded",
        "Origin": "https://ss.shipmentlink.com",
        "Pragma": "no-cache",
        "Referer": "https://ss.shipmentlink.com/tvs2/jsp/TVS2_InteractiveSchedule.jsp",
        "Sec-Fetch-Dest": "empty",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Site": "same-origin",
        "User-Agent": random_user_agent,
        "X-Requested-Call": "TUF1_AutoComplete",
        "X-Requested-With": "XMLHttpRequest",
        "sec-ch-ua": "\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Windows\""
    }
    url = "https://ss.shipmentlink.com/servlet/TUF1_AutoCompleteServlet"
    data = {
        "scope": "context",
        "search": query.upper(),
        "action": "preLoad",
        "switchSql": "",
        "datasource": "bkLocations",
        "fromFirst": "true"
    }
    # proxies = proxies_queue()
    res = requests_sys(url, headers=headers, data=data, POST=True)
    # res = proxy_requests_sys(url, headers=headers, data=data, POST=True, proxies=proxies)
    if res:
        # ip_queue.put(proxies)
        decode_text = decode(res.text)
        if decode_text:
            return decode_text
        else:
            return None
    elif res == 0:
        loss_requests += 1
        return None


origin_query = ''


def emc_transport(origin_city, destination_city):
    try:
        global total_requests
        global loss_requests
        headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "Content-Type": "application/x-www-form-urlencoded",
            "Origin": "https://ss.shipmentlink.com",
            "Pragma": "no-cache",
            "Referer": "https://ss.shipmentlink.com/tvs2/jsp/TVS2_InteractiveSchedule.jsp",
            "Sec-Fetch-Dest": "document",
            "Sec-Fetch-Mode": "navigate",
            "Sec-Fetch-Site": "same-origin",
            "Sec-Fetch-User": "?1",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": random_user_agent,
            "sec-ch-ua": "\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\"",
            "sec-ch-ua-mobile": "?0",
            "sec-ch-ua-platform": "\"Windows\""
        }
        current_date = datetime.datetime.now()
        departureYear = current_date.strftime("%Y")
        departureMonth = current_date.strftime("%m")
        departureDay = current_date.strftime("%d")
        arrivalDate = current_date.strftime("%Y%m")
        departureDateShow_part1 = current_date.strftime('%b').upper()
        departureDateShow_part2 = current_date.strftime('%Y')
        url = "https://ss.shipmentlink.com/tvs2/jsp/TVS2_InteractiveScheduleRouting.jsp"
        port_mapping = {}
        # file_path = "../maeu_ports.json"
        # file_path = "maeu_ports.json"
        # file_path = os.path.join(current_working_directory + r'/src/crawler', file_path)
        file_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'maeu_ports.json')
        if os.path.exists(file_path):
            with open(file_path, "r") as file:
                Menu_json = json.load(file)
        for info in Menu_json['ports']:
            port_mapping[info['portName'].upper()] = {
                'portCode': info.get('portCode'),
                'unLocationCode': info['unLocationCode'],
                'portName': info['portName'],
                "regionCode": info.get('regionCode')
            }
        origin_info = port_mapping.get(origin_city.upper())
        destination_info = port_mapping.get(destination_city.upper())
        if origin_info and destination_info:
            total_requests += 1
            origin_unLocationCode = origin_info['unLocationCode']
            des_unLocationCode = destination_info['unLocationCode']
            regionCode = destination_info.get('regionCode', None)
            if "TIANJIN" == origin_city.upper():
                origin_city = "XINGANG"
            origin_query = get_point(origin_city)[0]
            logger.info(f"{origin_city} ----> 查询成功: {origin_query}")
            destination_query = get_point(destination_city)
            destination_query_sys = None
            for info in destination_query:
                if des_unLocationCode.upper() == info[1].upper():
                    destination_query_sys = info
                    break
                elif des_unLocationCode.upper() in info[0]:
                    destination_query_sys = info
                    break
            logger.info(f"{destination_city} ----> 查询成功: {destination_query_sys}")
            if origin_query and destination_query_sys:
                data = {
                    "oriCountry": "",
                    "groupRadioOri": "ALL",
                    "desCountry": "",
                    "groupRadioDes": "ALL",
                    "departureMonth": departureMonth,
                    "departureDay": departureDay,
                    "departureYear": departureYear,
                    # "departureDate": str(departureYear) + str(departureMonth) + "01",
                    "departureDate": "",
                    "arrivalMonth": "",
                    "arrivalDay": "",
                    "arrivalYear": "",
                    "arrivalDate": f"{departureMonth}{departureDay}",
                    "durationWeek": "28",
                    "reeferCargo": "N",
                    "oriLocation": origin_query[1],
                    "oriLocationName": origin_query[0],
                    "desLocation": destination_query_sys[1],
                    "desLocationName": destination_query_sys[0],
                    "carrier": "V",
                    "serviceMode": "",
                    "isReefer": "N",
                    "func": "getSearchResult",
                    "oriUSCA": "",
                    "desUSCA": "",
                    "oriEastWest": "",
                    "desEastWest": "ALL",
                    "oriUseMode": "I",
                    "desUseMode": "I",
                    "departureDateShow": f"{departureDateShow_part1}-{departureDay}-{departureDateShow_part2}"
                }
                retry_count = 0
                while retry_count < 3:
                    proxies = proxies_queue()
                    # res = requests_sys(url, headers=headers, data=data, POST=True)
                    res = proxy_requests_sys(url, headers=headers, data=data, POST=True, proxies=proxies)
                    if res:
                        logger.info(f"当前查询 起点:{origin_city} ----> 终点: {destination_city}")
                        logger.info(f"当前请求总数: {total_requests}, 失败请求总数: {loss_requests}")
                        html = etree.HTML(res.text)
                        Departure_Date = html.xpath("//table/thead/tr[2]/td[2]/text()")
                        Arrival_Date = html.xpath("//table/thead/tr[2]/td[6]/text()")
                        Transit_Time = html.xpath("//table/thead/tr[1]/td[8]/text()")
                        Arrival_trans_place = html.xpath("//table/thead/tr[1]/td[6]/text()")
                        Arrival_place = html.xpath("//table/thead/tr[1]/td[7]/text()")
                        ship_list = html.xpath("//table/thead/tr[2]/td[4]/span/text()")
                        shipping_lanes = html.xpath("//table/thead/tr[1]/td[5]/span/text()")  # 航线编号
                        if Departure_Date:
                            ip_queue.put(proxies)
                            for departure_date, arrival, arrival_date, transit, ship, arrival_trans, shipping_lanes in zip(
                                    Departure_Date, Arrival_place, Arrival_Date, Transit_Time, ship_list,
                                    Arrival_trans_place, shipping_lanes):
                                item = {}
                                item['origin_code'] = f"{origin_info.get('portName').upper()}({origin_unLocationCode})"
                                if regionCode:
                                    item['destination_code'] = f"{destination_city}, {regionCode}({des_unLocationCode})"
                                else:
                                    item['destination_code'] = f"{destination_city}({des_unLocationCode})"
                                arrival = arrival.strip().upper().split(',')[0]
                                # item['destination_code'] = f"{port_mapping.get(arrival)['portName']}({port_mapping.get(arrival)['unLocationCode']})"
                                item['depart_date'] = datetime.datetime.strptime(departure_date.strip(),
                                                                                 "%b-%d-%Y").date().strftime("%Y-%m-%d")
                                if arrival_date and arrival_date.strip() != '----':
                                    item['arrive_date'] = datetime.datetime.strptime(arrival_date.strip(),
                                                                                     "%b-%d-%Y").date().strftime(
                                        "%Y-%m-%d")
                                else:
                                    transit_days = int(transit.strip()) if transit.strip().isdigit() else 0
                                    depart_date_obj = datetime.datetime.strptime(item['depart_date'], "%Y-%m-%d").date()
                                    arrive_date_obj = depart_date_obj + datetime.timedelta(days=transit_days)
                                    item['arrive_date'] = arrive_date_obj.strftime("%Y-%m-%d")
                                item['transport_contractor'] = "EMC"
                                item['haulage_time'] = transit.strip()
                                ship = ship.strip().split('\r\n')
                                item['ship_name'] = ship[0].strip()
                                item['voyage_number'] = ship[1].strip()
                                arrival_trans = arrival_trans.strip().upper().split(',')[0]
                                if destination_city.upper() != arrival_trans.upper():
                                    item[
                                        'transfer_code'] = f"{port_mapping.get(arrival_trans)['portName']}({port_mapping.get(arrival_trans)['unLocationCode']})"
                                if item.get('transfer_code'):
                                    if not port_mapping.get(item['transfer_code'].split('(')[0].upper()):
                                        del item['transfer_code']
                                item['shipping_lanes'] = shipping_lanes
                                print(item)
                                insert_shipping_info(item)

                            # print(len(Departure_Date))
                        else:
                            logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 未查询到数据")
                            read_transport_json(transport='EMC', origin_city=origin_city, des_city=destination_city,
                                                fail_result="未查询到数据")
                            loss_requests += 1
                            retry_count += 1
                        break
                    elif res == 0:
                        logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 未查询到数据, 请求超时")
                        retry_count += 1
                if retry_count == 3:
                    read_transport_json(transport='EMC', origin_city=origin_city, des_city=destination_city,
                                        fail_result="请求超时")
            else:
                logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 未查询到编号")
                read_transport_json(transport='EMC', origin_city=origin_city, des_city=destination_city,
                                    fail_result="未查询到编号")
                loss_requests += 1

    except Exception as e:
        logger.error(f"{origin_city} ----> {destination_city}: {e}")
        logger.error(traceback.format_exc())


def process_route(origin_city, destination_list):
    for destination_city in destination_list:
        emc_transport(origin_city, destination_city)


def emc_spider_run():
    global total_requests
    global loss_requests
    # origin_city_list = [
    #     "Shanghai", "XIAMEN", "NANSHA", "QINGDAO", "YANTIAN", "NINGBO"
    # ]
    # destination_list = [
    #     "Los Angeles", "Long Beach", "Atlanta", "Baltimore", "Boston", "Buffalo", "Calgary", "Charleston",
    #     "Chicago", "Chippewa Falls", "Cincinnati", "Cleveland", "Dallas", "Denver",
    #     "Detroit", "Edmonton", "EL Paso", "Halifax", "Houston", "Jacksonville",
    #     "Kansas City", "Louisville", "Memphis", "Miami", "Minneapolis", "Montreal",
    #     "Nashville", "New Orleans", "New York", "Norfolk", "Oakland", "Omaha",
    #     "Philadelphia", "Pittsburgh", "Portland", "Prince Rupert", "Saint Louis",
    #     "Salt Lake City", "San Antonio", "Saskatoon", "Savannah", "Seattle",
    #     "Tacoma", "Tampa", "Toronto", "Vancouver", "Winnipeg"
    # ]

    json_path_one = r"origin_and_destination.json"
    # json_path_one = r"origin_and_destination - 副本.json"
    json_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), json_path_one)
    with open(f'{json_path}', 'r', encoding='utf-8') as f:
        tmp = json.loads(f.read())

    origin_city_list = tmp['origin_city_list']
    destination_list = tmp['destination_list']
    del tmp, json_path, json_path_one

    # with concurrent.futures.ThreadPoolExecutor() as executor:
    #     futures = [executor.submit(process_route, origin_city, destination_list) for origin_city in origin_city_list]

    with concurrent.futures.ThreadPoolExecutor() as executor:
        futures = []
        for origin_city in origin_city_list:
            future = executor.submit(process_route, origin_city, destination_list)
            futures.append(future)
            time.sleep(1)

        for future in concurrent.futures.as_completed(futures):
            try:
                future.result()
            except Exception as e:
                # 输出异常详细信息，包括出错的行号
                print(f"An error occurred: {e}")
                print(traceback.format_exc())
    read_excel(transport="EMC", total_requests=total_requests, loss_requests=loss_requests)

<<<<<<< .mine

# if __name__ == '__main__':
#     emc_spider_run()
||||||| .r58155
if __name__ == '__main__':
    emc_spider_run()=======

if __name__ == '__main__':
    emc_spider_run()
>>>>>>> .r59648
